package com.omg.server.crawler.strategy.impl;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;

import net.htmlparser.jericho.Element;
import net.htmlparser.jericho.HTMLElementName;
import net.htmlparser.jericho.Source;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.omg.business.model.PostMeta;
import com.omg.business.model.Posts;
import com.omg.business.model.Terms;
import com.omg.business.model.TermsRelation;
import com.omg.business.model.TermsTaxonomy;
import com.omg.server.crawler.RequestContext;
import com.omg.server.crawler.strategy.CrawlerStrategy;
import com.omg.server.framework.service.IDefaultService;
import com.omg.server.util.CommonUtil;
import com.omg.server.util.DateUtil;
import com.omg.server.util.HttpSenderUtil;
import com.omg.weibo.WeiBoSender;

/**
 * 关于什么值得买 网站抓取策略
 * @author Administrator
 */
public class SmzdmStrategyImpl implements CrawlerStrategy{
	private static final Log log = LogFactory.getLog(SmzdmStrategyImpl.class);
	private IDefaultService defaultService;
	@Override
	public void handle(RequestContext requestContext) {
		int pageIndex = requestContext.getStartIndex();
		if (pageIndex < 1){
			return;
		}
		String url = requestContext.getSiteUrl();
		String soureUrl = requestContext.getSiteUrl();
		log.info(url + "第" + pageIndex + "抓取.");
		
		Source source = null;
		try {
			String subStract = null;
			String imgUrl = null;
			String imgUrl2 = null;
			String targetUrl = null;
			Element bizElement = null;
			Element subElement = null;
			String relativeUrl = null;
			Posts posts = null;
			PostMeta postMeta = null;
			PostMeta postMeta2 = null;
			List<Element> bizList = null;
			String price = null;
			String title = null;
			String issueTime = null;
			source = new Source(new URL(url+"page/"+pageIndex));
			// Call fullSequentialParse manually as most of the source will be parsed.
			source.fullSequentialParse();
			int currentYear = Calendar.getInstance().get(Calendar.YEAR);	
			int currentMon = Calendar.getInstance().get(Calendar.MONTH)+1;	
			int tmpMon = currentMon;
			String tmpValue = null;
			String pageUrl = null;
			String subStr1 = null;
			String subStr2 = null;
			List<Element> linkList = null;
			String linkContent = null;
			bizList = source.getAllElementsByClass("post show_all");
			Date issueDate = null;
			List<Integer> count = null;
			String type = null;
			String tag = null;
			Map<String, Object> params = new HashMap<String, Object>(2);
			HashSet<PostMeta> pmSet = new HashSet<PostMeta>(2);
			if (CommonUtil.isNotEmpty(bizList)){
				WeiBoSender wbs = new WeiBoSender();
				for (Element el : bizList){
					try{
						title = null;
						price = null;
						subStract = null;
						targetUrl = null;
						issueDate = null;
						type = null;
						imgUrl = null;
						imgUrl2 = null;
						issueTime = null;
						pageUrl = null;
						relativeUrl = null;
						posts = new Posts();
						postMeta = new PostMeta();
						postMeta2 = new PostMeta();
						subElement = el.getFirstElement(HTMLElementName.H2);
						if (null != subElement){
							subElement = subElement.getFirstElement(HTMLElementName.A);
							//分离出价格
							title = subElement.getTextExtractor().toString();
							
							subElement = subElement.getFirstElementByClass("title_end");
							if (null != subElement){
							    price = subElement.getTextExtractor().toString();
							    if (null != title){
							        title = title.replace(price, "");
							    }
							}
							posts.setPostTitle(title);
						}
						
						subElement = el.getFirstElementByClass("post_cat clearfix");
						if (null != subElement){
							if(CommonUtil.isNotEmpty(type)){
								type = new String(type.getBytes("GBK"), "UTF-8");
							}
						    type = subElement.getTextExtractor().toString();
						}
						
						subElement = el.getFirstElementByClass("post_date clearfix");
						if (null != subElement){
							//自动加年月日
							issueTime = subElement.getContent().toString();
							if (CommonUtil.isNotEmpty(issueTime)){
								issueTime = issueTime.trim();
								if (issueTime.indexOf(".") > -1){
									tmpValue = issueTime.substring(0, issueTime.indexOf("."));
									if(CommonUtil.isNotEmpty(tmpValue)){
										tmpMon = Integer.valueOf(tmpValue);
									}
									if(tmpMon > currentMon){
										currentYear = currentYear -1;
									}
								}
								issueTime = currentYear + "." + issueTime;
							}
							if (CommonUtil.isEmpty(issueTime)){
								continue;
							}
							issueDate = DateUtil.convertToDate(issueTime, DateUtil.YYYY_MM_DD_HHMM);
							if (null != requestContext.getMaxDate() && null != issueDate){
								if (issueDate.before(requestContext.getMaxDate())){
									continue;
								}
							}
							posts.setPostDate(issueDate);
							posts.setPostDate2(issueDate);
							posts.setPostModified(issueDate);
							posts.setPostModified2(issueDate);
						}
						
						
						subElement = el.getFirstElementByClass("post_tag");
						if (null != subElement){
							subElement = subElement.getFirstElement(HTMLElementName.A);
							if (null != subElement){
								tag = subElement.getTextExtractor().toString();
							}
						}
		
						subElement = el.getFirstElementByClass("post_thumb_wrap");
						if (null != subElement)
						    bizElement = subElement.getFirstElementByClass("buy_button");
						if (null != bizElement){
							bizElement = bizElement.getFirstElement(HTMLElementName.A);
							relativeUrl = bizElement.getAttributeValue("href");
							if (relativeUrl.startsWith("/") && relativeUrl.length() > 1){
							    subStr1 =  relativeUrl.substring(1, relativeUrl.lastIndexOf("/"));
							}
							else{
								 subStr1 =  relativeUrl.substring(0, relativeUrl.lastIndexOf("/"));
							}
							subStr2 =  relativeUrl.substring(relativeUrl.lastIndexOf("/")+1);
							if (subStr1.startsWith("http://")){
							    pageUrl = subStr1 + ".php?pid="+subStr2;
							}
							else{
								 pageUrl = url + subStr1 + ".php?pid="+subStr2;
							}
							
							targetUrl = HttpSenderUtil.getTargetUrl(pageUrl);//商品目标来源
							bizElement = subElement.getFirstElement(HTMLElementName.IMG);
							imgUrl = bizElement.getAttributeValue("src");//商品图片地址
							if (null !=imgUrl){
								if(imgUrl.indexOf("smzdm158.jpg") > 0){
									imgUrl = "images/logo.jpg";
								}
								else if(imgUrl.indexOf("http:") < 0){
									if (imgUrl.startsWith("/")){
									   imgUrl = soureUrl.substring(0, soureUrl.length()-1) + imgUrl;
									}
									else{
										imgUrl = soureUrl + imgUrl;
									}
								}
								
							}
							
							bizElement = el.getFirstElementByClass("p_excerpt");
							subStract = bizElement.getContent().toString();//商品摘要
							linkList = bizElement.getAllElements(HTMLElementName.A);
							if (CommonUtil.isNotEmpty(linkList) && null != subStract){
								for (Element cl : linkList){
									linkContent = cl.getTextExtractor().toString();
									subStract = subStract.replace(cl.toString(), linkContent);
								}
								subStract.replaceAll(soureUrl, "www.buymefan.com");
								List<Element> imgsList = bizElement.getAllElements(HTMLElementName.IMG);
								if(null != imgsList)
								for (Element e : imgsList){
									if (CommonUtil.isEmpty(imgUrl2)){
										imgUrl2 = e.getAttributeValue("src");
									}
									subStract = subStract.replace(e.toString(), "");
								}
							}
							
							if(CommonUtil.isEmpty(imgUrl)){
								imgUrl = imgUrl2;
							}
							targetUrl = CommonUtil.getFinalUrl(targetUrl);
							if (null != targetUrl){
								if (targetUrl.contains(soureUrl)){
									continue;
								}
								if (targetUrl.startsWith("ttp://")){
									targetUrl = "h" + targetUrl;
								}
							    //targetUrl = URLDecoder.decode(targetUrl);
							}
							//检查库中是否有这样的数据
							params.put("title", title);
							params.put("soureUrl", soureUrl);
							params.put("targetUrl", targetUrl);
							count = defaultService.find_iBatis("omg.crawler.checkGoodsIsExist", params);
							if (CommonUtil.isNotEmpty(count)){
								if (count.get(0) > 0){
									continue;
								}
							}
							//System.out.println("smzdm title:"+title);
							//System.out.println("smzdm subStract:"+subStract);
							//System.out.println("smzdm targetUrl:"+targetUrl);
							postMeta.setMetaKey("thumbnail");
							postMeta.setMetaValue(imgUrl);
							//postMeta.setId(UUID.randomUUID().toString());
							postMeta2.setMetaKey("targeturl");
							postMeta2.setMetaValue(targetUrl);
							//postMeta2.setId(UUID.randomUUID().toString());
							posts.setSourceUrl(url);
							posts.setPostContent(subStract);
							posts.setPostName(title);
							if(null != price && price.matches("[\\d.]+") && price.indexOf("元") < 0){
								price += "元";
							}
							posts.setPrice(price);
							
							if (CommonUtil.isNotEmpty(title) && CommonUtil.isNotEmpty(subStract) && CommonUtil.isNotEmpty(targetUrl)){
								pmSet.add(postMeta);
								pmSet.add(postMeta2);
								WriteDBUtil.writeToDb(posts, pmSet, type, tag, ", ", ", ", imgUrl, wbs);
								pmSet.clear(); //清空扩展
								log.info(url + "商品录入成功.");
							}
							log.info(url + "商品录入结束.");
					   }
				  }
				  catch (Exception e) {
						e.printStackTrace();
						log.error("录入商品失败", e);
				  }
				  finally{
					  pmSet.clear(); //清空扩展
				  }
			   }
			}
		}
		catch (MalformedURLException e) {
			e.printStackTrace();
			log.error("录入商品失败2", e);
		} 
		catch (IOException e) {
			e.printStackTrace();
		}
	}
	
    private void saveCataAndTag(String type, String tag, int postId){
    	Map<String, String> params = new HashMap<String,String>();
    	List<Integer> termTaxonomy = null;
    	int cataTermTaxonomyId = -1;
    	int tagTermTaxonomyId = -1;
    	
    	if (CommonUtil.isEmpty(type)){
			type = "其它";
		}
		else{
			type = type.trim();
		}
		
		
		String[] types = type.split(", ");
		TermsRelation termsRelation = null;
		TermsTaxonomy termsTaxonomy = null;
		Terms terms = null;
		for (int i=0, len=types.length; i < len; i++){
			cataTermTaxonomyId = -1;
			//检查目录是否存在
			params.clear();
			params.put("taxonomy", "category");
			params.put("termName", types[i]);
			termTaxonomy = defaultService.find_iBatis("omg.post.checkTermsIsExist", params);
			if (CommonUtil.isNotEmpty(termTaxonomy)){
				cataTermTaxonomyId = termTaxonomy.get(0);
			}
			
			//保存分类目录
			termsRelation = new TermsRelation();
			if (-1 == cataTermTaxonomyId){
				terms = new Terms();
				terms.setName(types[i]);
				terms.setSlug(URLEncoder.encode(types[i]));
				defaultService.save_Hibernate_Object(terms);
				termsTaxonomy = new TermsTaxonomy();
				termsTaxonomy.setTermId(terms.getId());
				termsTaxonomy.setTaxonomy("category");
				termsTaxonomy.setDescription("商品分类目录");
				defaultService.save_Hibernate_Object(termsTaxonomy);
				cataTermTaxonomyId = termsTaxonomy.getId();
			}
			termsRelation.setId(postId);
			termsRelation.setTermOrder(i);
			termsRelation.setTermTaxonomyId(cataTermTaxonomyId);
			defaultService.save_Hibernate_Object(termsRelation);
			log.debug("商品分类目录成功:" +postId);
		}
		
		if (CommonUtil.isNotEmpty(tag)){
			String[] tags = tag.split(", ");
			for (int i=0,len=tags.length;i < len; i++){
				
				tagTermTaxonomyId = -1;
				//检查标签是否已存在
				params.clear();
				params.put("taxonomy", "post_tag");
				params.put("termName", tags[i]);
				termTaxonomy = defaultService.find_iBatis("omg.post.checkTermsIsExist", params);
				if (CommonUtil.isNotEmpty(termTaxonomy)){
					tagTermTaxonomyId = termTaxonomy.get(0);
				}
				try{
					//保存标签
					termsRelation = new TermsRelation();
					if (-1 == tagTermTaxonomyId){
					   terms = new Terms();
					   terms.setName(tags[i]);
					   terms.setSlug(URLEncoder.encode(tags[i]));
					   defaultService.save_Hibernate_Object(terms);
					   termsTaxonomy = new TermsTaxonomy();
					   termsTaxonomy.setTermId(terms.getId());
					   termsTaxonomy.setTaxonomy("post_tag");
					   termsTaxonomy.setDescription("商品标签");
					   defaultService.save_Hibernate_Object(termsTaxonomy);
					   tagTermTaxonomyId = termsTaxonomy.getId();
					}
					termsRelation.setId(postId);
					termsRelation.setTermOrder(i);
					termsRelation.setTermTaxonomyId(tagTermTaxonomyId);
					defaultService.save_Hibernate_Object(termsRelation);
					log.debug("商品标签成功:" +postId);
				}
				catch(Exception e){
					e.printStackTrace();
				}
			}
		}
		
    }
	

	public IDefaultService getDefaultService() {
		return defaultService;
	}

	public void setDefaultService(IDefaultService defaultService) {
		this.defaultService = defaultService;
	}
	public static void main(String[] args){
		RequestContext requestContext = new RequestContext();
		requestContext.setStartIndex(1);
		requestContext.setSiteUrl("http://www.smzdm.com/");
		new SmzdmStrategyImpl().handle(requestContext);
	}
	
}
